package crawler;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.parser.ParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFRow;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileOutputStream;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Set;
import java.util.regex.Pattern;

public class CrawlerPage extends WebCrawler {

    private static final Logger logger = LoggerFactory.getLogger(CrawlerPage.class);

    final Pattern fileFilterPattern = Pattern.compile(".*\\.(zip|mp4|exe|css)\\?.*$",Pattern.CASE_INSENSITIVE);
    final Pattern NoDataPagePattern = Pattern.compile(".*(check.jsp|contact.jsp|yewufanchou.jsp|reg.jsp|/hyd/|chat.jsp|fazhanlishi.jsp|/lpai/|info.jsp).*$",Pattern.CASE_INSENSITIVE);
    final Pattern DataPagePattern = Pattern.compile(".*((.jsp)?\\?G=.*|(.jsp)?\\?T=.*\\&G=.*|(.jsp)?\\?P=.*\\&G=.*|(.jsp)?\\?OBY=.*\\&G=.*)",Pattern.CASE_INSENSITIVE);
    final static Pattern goodsPagePattern = Pattern.compile(".*(/xpai/goods).*\\.html$");

    FileOutputStream fileOut;

    final HSSFWorkbook workbook = new HSSFWorkbook();

    final HSSFSheet sheet = workbook.createSheet();

    static int rownum=0;

    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();
        logger.info("找到链接:{}",href);
        boolean machted =href.startsWith("http://www.huayicang.com")&&!fileFilterPattern.matcher(href).matches();
        //           &&!NoDataPagePattern.matcher(href).matches()&&!goodsPagePattern.matcher(href).matches();
        if(machted)
        {
            logger.info("=========匹配到url:{}",href);
        }
        return machted;
    }

    @Override
    public void visit(Page page) {
        String url = page.getWebURL().getURL();
        ParseData parseData = page.getParseData();
        if (!(parseData instanceof HtmlParseData)) return;
        logger.info("分析页面链接：{}", url);
        Set<WebURL> webURLS = parseData.getOutgoingUrls();
        for (WebURL webURL : webURLS) {
            String link = webURL.getURL();
            if (goodsPagePattern.matcher(link).matches()) {
                HSSFRow row = sheet.createRow(rownum++);
                logger.info("{}数据提取链接:{}",rownum,link);
                HSSFCell cell = row.createCell(0);
                cell.setCellValue(link);
            }
        }
    }

    @Override
    public void onStart() {
        SimpleDateFormat sdf=new SimpleDateFormat("yyyyMMddHHmmss");
        try {
            String filePath="d:/url" + sdf.format(new Date()) + ".xls";
            fileOut = new FileOutputStream(filePath);
            logger.info("创建数据文件：{}",filePath);
        }
        catch (Exception e)
        {

        }
    }

    @Override
    public void onBeforeExit() {
        try {
            workbook.write(fileOut);
            workbook.close();
            fileOut.close();
        }
        catch (Exception e)
        {
        }
    }


}
